{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content_id</th>\n",
       "      <th>content</th>\n",
       "      <th>subject</th>\n",
       "      <th>sentiment_value</th>\n",
       "      <th>sentiment_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>vUXizsqexyZVRdFH</td>\n",
       "      <td>因为森林人即将换代，这套系统没必要装在一款即将换代的车型上，因为肯定会影响价格。</td>\n",
       "      <td>价格</td>\n",
       "      <td>0</td>\n",
       "      <td>影响</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4QroPd9hNfnCHVt7</td>\n",
       "      <td>四驱价格貌似挺高的，高的可以看齐XC60了，看实车前脸有点违和感。不过大众的车应该不会差。</td>\n",
       "      <td>价格</td>\n",
       "      <td>-1</td>\n",
       "      <td>高</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>QmqJ2AvM5GplaRyz</td>\n",
       "      <td>斯柯达要说质量，似乎比大众要好一点，价格也低一些，用料完全一样。我听说过野帝，但没听说过你说...</td>\n",
       "      <td>价格</td>\n",
       "      <td>1</td>\n",
       "      <td>低</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>KMT1gFJiU4NWrVDn</td>\n",
       "      <td>这玩意都是给有钱任性又不懂车的土豪用的，这价格换一次我妹夫EP020可以换三锅了</td>\n",
       "      <td>价格</td>\n",
       "      <td>-1</td>\n",
       "      <td>有钱任性</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>nVIlGd5yMmc37t1o</td>\n",
       "      <td>17价格忒高，估计也就是14-15左右。</td>\n",
       "      <td>价格</td>\n",
       "      <td>-1</td>\n",
       "      <td>高</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         content_id                                            content  \\\n",
       "0  vUXizsqexyZVRdFH           因为森林人即将换代，这套系统没必要装在一款即将换代的车型上，因为肯定会影响价格。   \n",
       "1  4QroPd9hNfnCHVt7      四驱价格貌似挺高的，高的可以看齐XC60了，看实车前脸有点违和感。不过大众的车应该不会差。   \n",
       "2  QmqJ2AvM5GplaRyz  斯柯达要说质量，似乎比大众要好一点，价格也低一些，用料完全一样。我听说过野帝，但没听说过你说...   \n",
       "3  KMT1gFJiU4NWrVDn           这玩意都是给有钱任性又不懂车的土豪用的，这价格换一次我妹夫EP020可以换三锅了   \n",
       "4  nVIlGd5yMmc37t1o                            17价格忒高，估计也就是14-15左右。      \n",
       "\n",
       "  subject  sentiment_value sentiment_word  \n",
       "0      价格                0             影响  \n",
       "1      价格               -1              高  \n",
       "2      价格                1              低  \n",
       "3      价格               -1           有钱任性  \n",
       "4      价格               -1              高  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = \"train.csv\"\n",
    "df = pd.read_csv(path)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9947, 5)\n",
      "(8290,)\n"
     ]
    }
   ],
   "source": [
    "print(df.shape)\n",
    "print(df[\"content_id\"].unique().shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>-1</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>all</th>\n",
       "      <th>+_mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subject</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>价格</th>\n",
       "      <td>145</td>\n",
       "      <td>1014</td>\n",
       "      <td>114</td>\n",
       "      <td>1273</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>内饰</th>\n",
       "      <td>150</td>\n",
       "      <td>271</td>\n",
       "      <td>115</td>\n",
       "      <td>536</td>\n",
       "      <td>132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>动力</th>\n",
       "      <td>378</td>\n",
       "      <td>1970</td>\n",
       "      <td>384</td>\n",
       "      <td>2732</td>\n",
       "      <td>381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>外观</th>\n",
       "      <td>111</td>\n",
       "      <td>263</td>\n",
       "      <td>115</td>\n",
       "      <td>489</td>\n",
       "      <td>113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>安全性</th>\n",
       "      <td>93</td>\n",
       "      <td>380</td>\n",
       "      <td>100</td>\n",
       "      <td>573</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>操控</th>\n",
       "      <td>124</td>\n",
       "      <td>606</td>\n",
       "      <td>306</td>\n",
       "      <td>1036</td>\n",
       "      <td>215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>油耗</th>\n",
       "      <td>138</td>\n",
       "      <td>793</td>\n",
       "      <td>151</td>\n",
       "      <td>1082</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>空间</th>\n",
       "      <td>67</td>\n",
       "      <td>221</td>\n",
       "      <td>154</td>\n",
       "      <td>442</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>舒适性</th>\n",
       "      <td>256</td>\n",
       "      <td>564</td>\n",
       "      <td>111</td>\n",
       "      <td>931</td>\n",
       "      <td>183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>配置</th>\n",
       "      <td>154</td>\n",
       "      <td>579</td>\n",
       "      <td>120</td>\n",
       "      <td>853</td>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          -1     0    1   all  +_mean\n",
       "subject                              \n",
       "价格       145  1014  114  1273     129\n",
       "内饰       150   271  115   536     132\n",
       "动力       378  1970  384  2732     381\n",
       "外观       111   263  115   489     113\n",
       "安全性       93   380  100   573      96\n",
       "操控       124   606  306  1036     215\n",
       "油耗       138   793  151  1082     144\n",
       "空间        67   221  154   442     110\n",
       "舒适性      256   564  111   931     183\n",
       "配置       154   579  120   853     137"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnt_data = []\n",
    "for k, gp in df.groupby(\"subject\"):\n",
    "    cnt = gp.groupby(\"sentiment_value\").count()[\"content\"].to_dict()\n",
    "    cnt[\"subject\"] = k\n",
    "    cnt_data.append(cnt)\n",
    "dfcnt = pd.DataFrame(cnt_data)[[\"subject\", -1, 0, 1]]\n",
    "dfcnt[\"all\"] = dfcnt[-1] + dfcnt[1] + dfcnt[0]\n",
    "dfcnt[\"+_mean\"] = ((dfcnt[-1] + dfcnt[1]) / 2).map(int)\n",
    "dfcnt.index = dfcnt[\"subject\"]\n",
    "del dfcnt[\"subject\"]\n",
    "dfcnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "zeros_num = dfcnt[\"+_mean\"].to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "init_len: 3286\n",
      "0_len: 6661\n",
      "-----------------------\n",
      "1014 129 129\n",
      "271 132 132\n",
      "1970 381 381\n",
      "263 113 113\n",
      "380 96 96\n",
      "606 215 215\n",
      "793 144 144\n",
      "221 110 110\n",
      "564 183 183\n",
      "579 137 137\n",
      "-----------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(4926, 5)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 对各个主题的sentiment_value 为0的样本进行降采样\n",
    "balance = [df.query(\"sentiment_value!=0\")]\n",
    "print(\"init_len:\", len(balance[0]))\n",
    "print(\"0_len:\", len( df.query(\"sentiment_value==0\")))\n",
    "print(\"-----------------------\")\n",
    "for k, gp in df.query(\"sentiment_value==0\").groupby(\"subject\"):\n",
    "    sp = gp.sample(zeros_num[k])\n",
    "    balance.append(sp)\n",
    "    print(len(gp), len(sp), zeros_num[k])\n",
    "df_balance = pd.concat(balance, axis=0)\n",
    "print(\"-----------------------\")\n",
    "df_balance.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 合并主题标签和情感值标签为一个标签\n",
    "df = df_balance\n",
    "df[\"label\"] = df[\"subject\"] + \":\" + df[\"sentiment_value\"].map(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "for k, gp in df.groupby(\"content_id\"):\n",
    "    y = gp[\"label\"].tolist()\n",
    "    x = gp[\"content\"].values[0]\n",
    "    data.append((x, y))\n",
    "\n",
    "random.shuffle(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X_0, train_y_0 = zip(*data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectKBest, SelectPercentile\n",
    "from sklearn.feature_selection import chi2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 多标签的处理\n",
    "lb = MultiLabelBinarizer()\n",
    "y = lb.fit_transform(train_y_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba\n",
    "import re\n",
    "def process_docs(docs, cut_all=True):\n",
    "    new_docs = []\n",
    "    for doc in docs:\n",
    "        new_doc = \" \".join(list(filter(lambda x:bool(re.search(\"\\w\", x)),  jieba.cut(doc,cut_all=cut_all))))\n",
    "#         new_doc = re.sub(\"\\s\\d+\\s\", \" NUM \", new_doc)\n",
    "        new_docs.append(new_doc)\n",
    "    return new_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#new_train_x = process_docs(train_X_0)\n",
    "\n",
    "# vec=  TfidfVectorizer(token_pattern=r\"(?u)\\b\\S+\\b\",\n",
    "#                                    stop_words=[],\n",
    "#                                    max_df=0.6, \n",
    "#                                    min_df=5)\n",
    "\n",
    "# X = vec.fit_transform(new_train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/utils/__init__.py:93: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
      "  if np.issubdtype(mask.dtype, np.int):\n"
     ]
    }
   ],
   "source": [
    "vec = CountVectorizer(binary=True,\n",
    "                                    token_pattern=r\"(?u)\\b\\S+\\b\",\n",
    "                                    ngram_range=(1, 4), \n",
    "                                    analyzer=\"char\",\n",
    "                                   max_df=0.6, \n",
    "                                   min_df=5)\n",
    "\n",
    "# X = vec.fit_transform([re.sub(\"\\W\",\"\", doc) for doc in train_X_0])\n",
    "X = vec.fit_transform(train_X_0)\n",
    "#select = SelectKBest(chi2, k=150)\n",
    "select = SelectPercentile(chi2, 70)\n",
    "X=select.fit_transform(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "...     X, y, test_size=0.25, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 多标签分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 采用 OneVsRestClassifier\n",
    "# from sklearn.multiclass import OneVsRestClassifier\n",
    "# from sklearn import svm\n",
    "# from sklearn.model_selection import KFold, cross_val_score\n",
    "\n",
    "# clf =  OneVsRestClassifier(estimator=svm.SVC(kernel='linear'))\n",
    "# k_fold = KFold(n_splits=8)\n",
    "# scores = cross_val_score(clf, train_X, train_y, cv=k_fold, n_jobs=-1)\n",
    "# print(scores)\n",
    "\n",
    "# clf.fit(X_train, y_train)\n",
    "# clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 采用  MLkNN\n",
    "from skmultilearn.adapt import MLkNN\n",
    "from sklearn.metrics import accuracy_score "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = MLkNN(k=20, s=1)\n",
    "# train\n",
    "classifier.fit(X_train, y_train)\n",
    "# predict\n",
    "predictions = classifier.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9256535947712419\n"
     ]
    }
   ],
   "source": [
    "z = predictions.toarray()\n",
    "[rows, cols] = y_test.shape\n",
    "zzz = 0 \n",
    "for i in range(rows - 1):\n",
    "    for j in range(cols - 1):\n",
    "        if y_test[i, j] == z[i, j]:\n",
    "            zzz += 1 \n",
    "acc = zzz / (rows*cols)\n",
    "print(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLkNN(ignore_first_neighbours=0, k=20, s=1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = MLkNN(k=20, s=1)\n",
    "clf.fit(X, y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = [ \"就是呀，觉得对不起这个价格。开着烦心 \"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pipe_clf =  Pipeline([(\"tfidfVectorizer\",vec), (\"clf\", classifier)])\n",
    "# sample=process_docs(test)\n",
    "# rs = pipe_clf.predict(sample)\n",
    "# print(lb.inverse_transform(rs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('价格:-1',)]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/python3/lib/python3.6/site-packages/sklearn/utils/__init__.py:93: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
      "  if np.issubdtype(mask.dtype, np.int):\n"
     ]
    }
   ],
   "source": [
    "pipe_clf =  Pipeline([(\"tfidfVectorizer\",vec),(\"sel\",select), (\"clf\", clf)])\n",
    "# rs = pipe_clf.predict([re.sub(\"\\W\",\"\", doc) for doc in test])\n",
    "rs = pipe_clf.predict(test)\n",
    "print(lb.inverse_transform(rs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['clf.m']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(pipe_clf, \"models/clf.m\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['multi_label_binarizer.m']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(lb, \"models/multi_label_binarizer.m\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
